{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "# train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop(['interest_level'], axis=1, inplace = False)\n",
    "X_train = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.2,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9870, 227)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_part.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.69662, std: 0.00998, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.69280, std: 0.01335, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.68968, std: 0.01068, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.68623, std: 0.01222, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.68477, std: 0.01137, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.68299, std: 0.01073, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 2},\n",
       " -0.6829855586641879)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=103,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train_part , y_train_part)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([353.64886146, 354.65716634, 354.45759697, 354.83658218,\n",
       "        353.23407001, 322.31099129]),\n",
       " 'std_fit_time': array([ 1.36369097,  1.01033766,  0.98912165,  0.41269217,  0.26031049,\n",
       "        39.68024529]),\n",
       " 'mean_score_time': array([1.07013803, 1.07851706, 1.12937956, 1.06834345, 1.10963254,\n",
       "        0.88702798]),\n",
       " 'std_score_time': array([0.0128182 , 0.00704001, 0.09738297, 0.00919366, 0.0296712 ,\n",
       "        0.22015327]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'split0_test_score': array([-0.69798895, -0.7072186 , -0.69369706, -0.69572566, -0.69423495,\n",
       "        -0.69150166]),\n",
       " 'split1_test_score': array([-0.71461596, -0.70923381, -0.70809747, -0.7048595 , -0.70201849,\n",
       "        -0.69764162]),\n",
       " 'split2_test_score': array([-0.68544588, -0.67785481, -0.67794104, -0.67251431, -0.67650219,\n",
       "        -0.6717914 ]),\n",
       " 'split3_test_score': array([-0.68985777, -0.67924266, -0.68114917, -0.67604236, -0.67231471,\n",
       "        -0.67022509]),\n",
       " 'split4_test_score': array([-0.69520922, -0.69043919, -0.68749591, -0.68202147, -0.67877137,\n",
       "        -0.6837641 ]),\n",
       " 'mean_test_score': array([-0.69662384, -0.69279951, -0.68967676, -0.68623405, -0.68476991,\n",
       "        -0.68298556]),\n",
       " 'std_test_score': array([0.00998306, 0.01334768, 0.01068358, 0.01222197, 0.01137187,\n",
       "        0.01073411]),\n",
       " 'rank_test_score': array([6, 5, 4, 3, 2, 1]),\n",
       " 'split0_train_score': array([-0.12462314, -0.12852348, -0.13455866, -0.14194388, -0.14522289,\n",
       "        -0.15073778]),\n",
       " 'split1_train_score': array([-0.12568463, -0.1276402 , -0.13428448, -0.14138187, -0.14550242,\n",
       "        -0.15080407]),\n",
       " 'split2_train_score': array([-0.12765112, -0.13246527, -0.13802106, -0.14561194, -0.14783412,\n",
       "        -0.15429778]),\n",
       " 'split3_train_score': array([-0.1268453 , -0.13055293, -0.13708697, -0.14459303, -0.14717166,\n",
       "        -0.15362578]),\n",
       " 'split4_train_score': array([-0.12729936, -0.1310345 , -0.13682793, -0.14422783, -0.14752354,\n",
       "        -0.15305807]),\n",
       " 'mean_train_score': array([-0.12642071, -0.13004328, -0.13615582, -0.14355171, -0.14665093,\n",
       "        -0.1525047 ]),\n",
       " 'std_train_score': array([0.00111708, 0.00174315, 0.00147314, 0.00161737, 0.00107619,\n",
       "        0.00146917])}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.682986 using {'reg_alpha': 2, 'reg_lambda': 2}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOX1+PHPyb4QkpCEJYQkLGFfQogi4gJaFFxqrdYqlbZaobXFtaXqt7VS/frTtq4tbqgU9eu+Ia5AKyAqKGHfZN8CCIhJ2EOW8/vj3iSTkI1MJjNJzvv1mheZe5+5cy4wOXPu89znEVXFGGOMaaggfwdgjDGmebNEYowxxiuWSIwxxnjFEokxxhivWCIxxhjjFUskxhhjvGKJxBhjjFcskRhjjPGKJRJjjDFeCfF3AE0hMTFR09PT/R2GMcY0K0uWLPlOVZPqatcqEkl6ejo5OTn+DsMYY5oVEdlen3Z2acsYY4xXLJEYY4zxiiUSY4wxXmkVfSTGmNoVFRWRm5vL8ePH/R2K8YOIiAhSUlIIDQ1t0OstkRhjyM3NJSYmhvT0dETE3+GYJqSqHDhwgNzcXLp27dqgY9ilLWMMx48fJyEhwZJIKyQiJCQkeFWNWiIxxgBYEmnFvP23t0RSi7nr97Fg435/h2GMMQHN+khq8cz8zSza8j1nZyRy55je9EuO9XdIxhgTcKwiqcUL15/Ony/uw8rcAi751+fc/sZyduUf83dYxrQ4+fn5PPnkkw167WOPPcbRo0cbOaKG27ZtG/379/e6TW0OHDjAyJEjadOmDRMnTqyx3eTJk+ncuTOZmZlkZmby0UcfNfg9a2OJpBbhIcHccHY3Pps0kgnndOODlXsY+dA8HvhoHQVHi/wdnjEtRlMmkpKSkga9TyCJiIjgvvvu46GHHqqz7W233cby5ctZvnw5F110kU/isUtb9RAbFcpdY/rw82HpPDJ7A1MXbOG1xTu56bwejBuWRnhIsL9DNKbR/PX9NazdfbBRj9k3uS33XNqvxv133nknmzdvJjMzk1GjRtG+fXveeOMNCgsLufzyy/nrX//KkSNHuOqqq8jNzaWkpIS7776bvXv3snv3bkaOHEliYiJz586t9vht2rTh9ttvZ9asWTz88MNERkZy++23c/jwYRITE5k+fTqdOnVi8eLF/OpXvyI6OpqzzjqLjz/+mNWrV1d7zG3btjFu3DiOHDkCwJQpUzjzzDMrtZk+fTrvvvsuhYWFbN26lbFjx3LPPfcATkIbP348X375JZ07d+a9994jMjKSZ599lqlTp3LixAl69OjBSy+9RFRUVKXjlsW3adOmev8b+JJVJKegc1wkD181iA9vOptBXeL43w/Xcf7D83lv+S5KS9Xf4RnTbD344IN0796d5cuXM2rUKDZu3MjXX3/N8uXLWbJkCZ999hmffPIJycnJrFixgtWrVzN69GhuvvlmkpOTmTt3bo1JBODIkSP079+fr776iqFDh3LTTTfx1ltvsWTJEq6//nr+9Kc/AXDdddfx9NNPs3DhQoKDa/+C2L59e+bMmcPSpUt5/fXXufnmm6tt9/XXX/Pyyy+zfPly3nzzzfIJZDdu3Mjvfvc71qxZQ1xcHG+//TYAP/7xj1m8eDErVqygT58+PP/88wDMnDmTv/zlL6f8dztlyhQGDhzI9ddfT15e3im/vj6sImmAvsltefH601mwcT8PfPQNt7y2nGcXbOF/xvThzB6J/g7PGK/UVjk0hdmzZzN79mwGDx4MwOHDh9m4cSNnn302f/jDH7jjjju45JJLOPvss+t9zODgYK644goA1q9fz+rVqxk1ahTgVAadOnUiPz+fQ4cOlVcVY8eO5YMPPqjxmEVFRUycOJHly5cTHBzMhg0bqm03atQoEhISACdJfP755/zoRz+ia9euZGZmAjBkyBC2bdsGwOrVq/nzn/9Mfn4+hw8f5sILLwTghz/8IT/84Q/rfc4AN954I3fffTciwt13383vf/97pk2bdkrHqA+fJhIRGQ08DgQDz6nqg9W0uQqYDCiwQlXHutv/BlzsNrtPVV93twvwv8BPgBLgKVX9py/PoyZnZyQx/KZE3luxi4dmbWDsc18xolcSd47pTe+Obf0RkjHNnqpy11138etf//qkfUuWLOGjjz7irrvu4oILLqj3N/SIiIjyCkNV6devHwsXLqzU5lS/rT/66KN06NCBFStWUFpaSkRERLXtqt6jUfY8PDy8fFtwcDDHjjkDeX75y18yY8YMBg0axPTp05k3b94pxeWpQ4cO5T+PHz+eSy65pMHHqo3PLm2JSDDwBDAG6AtcIyJ9q7TJAO4ChqtqP+BWd/vFQBaQCQwFJolI2W/mXwJdgN6q2gd4zVfnUB9BQcLlg1P47+/P5U8X9WHp9jzGPL6AP7y5gj0FNsLLmPqIiYnh0KFDAFx44YVMmzaNw4cPA7Br1y727dvH7t27iYqK4tprr+UPf/gDS5cuPem19dGrVy/2799fnkiKiopYs2YN8fHxxMTEsGjRIgBee632Xy0FBQV06tSJoKAgXnrppRo78efMmcP333/PsWPHmDFjBsOHD6/1uIcOHaJTp04UFRXx8ssv1/u8qrNnz57yn999912vRorVxpcVyenAJlXdAiAirwGXAWs92owHnlDVPABV3edu7wvMV9VioFhEVgCjgTeAG4Gxqlpa5TV+FREazPhzuvGT7BSenLeZ6V9s4/0Vu7n+rK7cOKI7bSMaNhmaMa1BQkICw4cPp3///owZM4axY8cybNgwwOko/7//+z82bdrEpEmTCAoKIjQ0lKeeegqACRMmMGbMGDp16lRrP0mZsLAw3nrrLW6++WYKCgooLi7m1ltvpV+/fjz//POMHz+e6OhoRowYQWxszfeO/fa3v+WKK67gzTffZOTIkURHR1fb7qyzzmLcuHFs2rSJsWPHkp2dXX4Zqzr33XcfQ4cOJS0tjQEDBpQnyZkzZ5KTk8O9994LOAv2HTx4kBMnTjBjxgxmz55N3759ueGGG/jNb35DdnY2f/zjH1m+fDkiQnp6Os8880ydfz8NIaq+6SQWkSuB0ap6g/t8HDBUVSd6tJkBbACG41z+mqyqn4jIBcA9wCggCvgaJ+E8LCIHgEeAy4H9wM2qurGa958ATABITU0dsn17vRb6ajQ7vz/KI3M28O6yXcRHhXLTeRn87IxUG+FlAtK6devo06ePv8Pwu8OHD9OmTRvAGQCwZ88eHn/88QYfb/r06eTk5DBlypTGCtFnqvs/ICJLVDW7rtf6ctRWdZO3VM1aIUAGMAK4BnhOROJUdTbwEfAl8CqwECh2XxMOHHdP7lmg2p4jVZ2qqtmqmp2UVOeSw42uS7soHv1pJh/cdBZ9k9ty7wdr+cEj83l/xW4b4WVMgPrwww/JzMykf//+LFiwgD//+c/+DqlZ8GVFMgynwrjQfX4XgKo+4NHmaWCRqk53n/8XuFNVF1c51ivA/6nqRyLyDU6ls83teM9X1VrnLsnOzlZ/rtmuqny28Tse+Ggd33x7iIEpsdw1pg/Duif4LSZjPLWUimTo0KEUFhZW2vbSSy8xYMCABh9z1qxZ3HHHHZW2de3alXfffbfBxwxE3lQkvuwjWQxkiEhXYBdwNTC2SpsZOJXIdBFJBHoCW9yO+jhVPSAiA4GBwGyP15yHU4mci3NpLKCJCOf2TOKsHonMWLaLh2ev55pnF3Fe7/bcOaY3PTvE+DtEY1qEr776qtGPeeGFF5YPwTXV81kiUdViEZkIzMLp/5imqmtE5F4gR1VnuvsuEJG1OEN5J7nJIwJY4A6TOwhc63a8AzwIvCwitwGHgRt8dQ6NLThIuGJIChcP7MT0L7fxxNxNjH7sM34ypAu3jepJx9jqhw8aY0wg89mlrUDi70tbNck7coIpczfx4sJtBAcJN5zVjV+f240YG+FlmlhLubRlGi5QO9tNHeKjw7j7kr58+vsRXNC3I1PmbuLcf8xj+hdbOVFc6u/wjDGmXiyRBIAu7aL45zWDeX/iWfTqEMPk99cy6tH5fLhyD62hYjTGNG+WSALIgJRYXhk/lH9fdxoRIcH87pWl/OjJL/lqywF/h2aMT9l6JKdmzpw5DBkyhAEDBjBkyBA+/fTTBh+rMVgiCTAiwshe7fnolrP5+5UD2VtwnJ9OXcQNL+SwaV/9p4Ewpjmx9UhOTWJiIu+//z6rVq3ihRdeYNy4cX6Nx2b/DVDBQcJV2V24dGAy//5yK0/N3cwFj37GT0/rwm0/6En7tjbCy/jIx3fCt6sa95gdB8CYk+ZsLWfrkZzaeiRlMyMD9OvXj+PHj1NYWFhpIsimZBVJgIsMC+a3I3ow/48j+cWZ6by1JJdz/zGPR2av53Bhcd0HMKYZsPVIGr4eydtvv83gwYP9lkTAKpJmo110GPdc2o9fnpnOP2at55+fbuLlr3Zw6w8yuPr0VEKD7TuBaSS1VA5NwdYjqf96JGvWrOGOO+5g9uzZ+JMlkmYmLSGaKWOzuOHsfB74aB13v7eGaV9s448X9mJ0/44nrX1gTHNj65HUbz2S3NxcLr/8cl588UW6d+9+SrE3Nvsa20xldonjtQln8PwvsgkJEm58eSlXPPUli7d97+/QjDllth5JhfqsR5Kfn8/FF1/MAw88UOfxmoIlkmZMRDi/Twc+vuVsHvzxAHLzjvGTpxcy4cUcNu077O/wjKk3z/VI5syZU74eyYABA7jyyis5dOgQq1at4vTTTyczM5P777+/fGbesvVIRo4cWa/3KluP5I477mDQoEFkZmby5ZdfAvD8888zYcIEhg0bhqrWuR7JCy+8wBlnnMGGDRvqXI8kMzOTK664guzs2m8UL1uPZNSoUfTu3bt8u2cfyZQpU9i0aRP33XcfmZmZZGZmsm+f/5ZmsilSWpCjJ4qZ9vlWnp6/hWNFJVx9Whdu+UEG7WNshJepnU2R4rD1SAJv9l/TxKLCQph4ntP5/q//buTlr3bw7rJdTDinG+PP7kZ0uP1zG1ObDz/8kAceeIDi4mLS0tKYPn26v0NqFqwiacG2fneEh2at58NVe0hsE86tP8jgp6d1sRFe5iQtpSKx9UgazpuKxBJJK7B0Rx4PfLSOxdvy6JYUzR2je3NB3w42wsuUW7duHb1797b/E62UqvLNN9/Y7L+mZlmp8bzx62E8+/NsBPj1S0v4ydMLWbL91IY7mpYrIiKCAwcO2CShrZCqcuDAgRqHL9eHVSStTHFJKW/k5PLofzaw/1AhY/p3ZNKFveiW1MbfoRk/KioqIjc3l+PHj/s7FOMHERERpKSkEBpaeS0ku7TlwRLJyY4UFvPcgq1M/WwzhcWlXHN6Kjefn0FSjP+mWTDGBBZLJB4skdRs/6FC/vnfjbzy9Q4iQoL49bndueHsrkSF2QgvY1o7SyQeLJHUbfP+w/zjk/V8suZbkmLCuX1UT34yJIUQG+FlTKtlne3mlHRPasPT44bw9o3DSG0XxV3vrGL04wuYs3avdcAaY2rl00QiIqNFZL2IbBKRO2toc5WIrBWRNSLyisf2v4nIavfx02pe9y8RsXlAGtmQtHa89ZthPH3tEEpLlfEv5vDTZxaxbIeN8DLGVM9nF8JFJBh4AhgF5AKLRWSmqq71aJMB3AUMV9U8EWnvbr8YyAIygXBgvoh8rKoH3f3ZQJyvYm/tRITR/Ttyfp/2vL54J4/9ZwOXP/klFw/oxKQLe5GeWP2cQsaY1smXFcnpwCZV3aKqJ4DXgMuqtBkPPKGqeQCqWjbrWF9gvqoWq+oRYAUwGsoT1D+AP/owdgOEBgdx7RlpzJs0klvOz+DTb/bxg0fmM3nmGg4cLqz7AMaYVsGXiaQzsNPjea67zVNPoKeIfCEii0RktLt9BTBGRKJEJBEYCXRx900EZqrqntreXEQmiEiOiOTs37/f65NpzdqEh3DbqJ7MnzSCq07rwkuLtnPuP+bxxNxNHDvR/Ne/NsZ4x5eJpLq5Fqr22oYAGcAI4BrgORGJU9XZwEfAl8CrwEKgWESSgZ8A/6rrzVV1qqpmq2p2UlJSw8/ClGvfNoL/d/kAZt16NsO6J/CPWesZ8dBcXl+8g5JS65A3prXyZSLJpaKKAEgBdlfT5j1VLVLVrcB6nMSCqt6vqpmqOgonKW0EBgM9gE0isg2IEpFNPjwHU40e7WN49ufZvPmbYSTHRXLH26sY8/hnfPqNjfAypjXyZSJZDGSISFcRCQOuBmZWaTMD57IV7iWsnsAWEQkWkQR3+0BgIDBbVT9U1Y6qmq6q6cBRVe3hw3MwtTgtvR3v3HgmT/0sixPFpVw/PYdrnl3Eip35/g7NGNOEfDZqS1WLRWQiMAsIBqap6hoRuRfIUdWZ7r4LRGQtUAJMUtUDIhIBLHBnIj0IXKuqxb6K1TSciDBmQCd+0LcDr369g8f/s5HLnviCSwclM+mCXqQmRPk7RGOMj9md7aZRHTpexNTPtvDsgi2UlCrXnpHGTedl0C46zN+hGWNOkU2R4sESSdPbe/A4j87ZwBs5O4kOC+HGkd25fnhXIkKD/R2aMaaebIoU41cd2kbw4BUDmXXrOQzt1o6/f7KekQ/N482cnTbCy5gWxhKJ8amMDjE894vTeG3CGbSPCWfSWyu5+J8LmLt+n43wMqaFsERimsQZ3RKY8bvhTBk7mKMnSrju34v52XNfsSq3wN+hGWO8ZInENBkR4ZKByfzn9nOZfGlf1u05yKVTPueW15ax8/uj/g7PGNNA1tlu/Obg8SKemb+Z5xZsRRV+PiyNief1IC7KRngZEwhs1JYHSySBbU/BMR6ds4E3l+QSEx7C70b24BdnptsIL2P8zEZtmWajU2wkf79yEB/fcjZD0uJ54ONvOO+heby9JJdSG+FlTMCzRGICRu+Obfn3dafzyg1DSWgTzu/fXMHF//qczzbY7M3GBDJLJCbgnNkjkfd+N5zHr87kcGERP5/2NeOe/4o1u22ElzGByBKJCUhBQcJlmZ35z+3ncvclfVm1q4BL/vU5t7++nNw8G+FlTCCxznbTLBQcK+KpeZuZ9sVWSkuV/p1jyUqNJystjqzUeJLjIv0dojEtjo3a8mCJpOXYnX+MlxZtZ8m2PFbk5lNYXApAx7YRDEmLZ3BqHFlp8fRLbkt4iI36MsYb9U0kPptG3hhfSI6L5I7RvQE4UVzKuj0HWbojj6U78lm6PY8PVzkrMIcFB9G/c1u3aoknKzWejrER/gzdmBbLKhLTouw7eLxSYlm5q4ATbtWSHBtRnlSy0uLp26ktYSHWTWhMTezSlgdLJK3XieJS1uwucBLLjjyWbc9jd8FxAMJDghjQOdZNLk5fS/u2VrUYU8YSiQdLJMbTtwVu1bI9j6U78li96yAnSpyqpXNcJEPKEktaPH06tSU02KoW0zpZIvFgicTUprC4hNW7DrJsR56bYPL59qBTtUSEBjGwcxyD3dFhWanxJMWE+zliY5qGJRIPlkjMqdqdf6w8qSzdkcea3QUUlTiflS7tIslKjXcrl3h6d4whxKoW0wLZqC1jvJAcF0lyXCSXDEwG4HhRCWt2F7Bku5NcFm4+wHvLdwMQGRrMwJTYio781DgS2ljVYloPnyYSERkNPA4EA8+p6oPVtLkKmAwosEJVx7rb/wZc7Da7T1Vfd7e/DGQDRcDXwK9VtciX52FMRGgwQ9LaMSStHQCqyq78Y+Wjw5btyOPZz7ZQ7E4ymZYQVZ5UstLi6dXBqhbTcvns0paIBAMbgFFALrAYuEZV13q0yQDeAM5T1TwRaa+q+0TkYuBWYAwQDsx32xwUkYuAj91DvAJ8pqpP1RaLXdoyTeF4UQmrdhWwdHueU7nsyOe7w4UARIUFMyglrvxO/MGp8bSLtnVXTGALhEtbpwObVHWLG9BrwGXAWo8244EnVDUPQFX3udv7AvNVtRgoFpEVwGjgDVX9qOzFIvI1kOLDczCm3iJCgzktvR2npVdULbl5xzxGiOXz9PwtlLhVS9fEaOdOfLe/pWeHGIKDxJ+nYEyD+DKRdAZ2ejzPBYZWadMTQES+wLn8NVlVPwFWAPeIyCNAFDCSygkIEQkFxgG3VPfmIjIBmACQmprq7bkYc8pEhC7toujSLorLMjsDcOxECStz88vva5m/fj/vLN0FQJvwEAZ1iS0fHTY4Nc5WizTNgi8TSXVfrapeRwsBMoAROJXFAhHpr6qzReQ04EtgP7AQKK7y2idxLmstqO7NVXUqMBWcS1sNPQljGlNkWDBDuyUwtFsC4FQtO74/WmmE2JPzNpdXLd2SossTy5C0eDLatyHIqhYTYHyZSHKBLh7PU4Dd1bRZ5HaWbxWR9TiJZbGq3g/cDyAirwAby14kIvcAScCvfRe+Mb4nIqQlRJOWEM3lg52rtEdPFLNiZ4FzJ/6OPD79Zh9vLckFICY8hMzUOAa7HfmDu8QTGxXqz1MwxqeJZDGQISJdgV3A1cDYKm1mANcA00UkEedS1xa3oz5OVQ+IyEBgIDAbQERuAC4EzlfVUh/Gb4xfRIWFMKx7AsO6V1Qt2w4cLb8Tf+mOfKZ8upGyVYh7tG9TPsXLkLR4uidZ1WKals8SiaoWi8hEYBZO/8c0VV0jIvcCOao60913gYisBUqASW7yiMC5zAVwELjW7XgHeBrYDix097+jqvf66jyM8TcRoWtiNF0To7liiFO1HC4sZuXO/PLEMnvtXt7IcaqWthEhZKZWzB+WmRpH2wirWozv2J3txrQAqsqW746Ujw5btiOP9XsPoQoikNG+TXlfS1ZaHN0SrWoxdbMpUjxYIjGt0aHjReV9LU5/Sz4Fx5x7d2MjQ8uHHmelxjOoSywxVrWYKgLhPhJjjB/FRIRyVkYiZ2UkAlBa6lYtHjMfz9+wv7xq6dUhprwTPystnm6J0biXj42plVUkxrRiB48XsXxHRV/Lsh15HDrudEfGRYVWTPOSGs+gLnFEh9t3z9bEKhJjTJ3aRoRyTs8kzumZBDhVy+b9h1m6o2Kal0+/cSacCBLo1bFteWLJSosnPSHKqhZjFYkxpnYFR4tYtrOiYlm+I59DhU7V0i46zLmfxaOvJSrMvp+2FFaRGGMaRWxUKCN6tWdEr/YAlJQqm/YdLu9rWbIjj/+sc6qW4CChd8eY8tFhWanxpLazqqWlq7MiEZHuQK6qForICJybA19U1fwmiK9RWEVijG/lHz3BsvK+FqdqOXKiBIDENmHlFUtWahwDU+KIDAv2c8SmPhpt+K+ILMdZ/yMd5wbCmUAvVb2oEeJsEpZIjGlaJaXKhr2HyucQW7Yjjy3fHQEgJEjo06lt+eiwrNR4UuIjrWoJQI2ZSJaqapaITAKOq+q/RGSZqg5urGB9zRKJMf73/ZETLHMrlqXb81mRm8/R8qolnKzUOGf54rR4BnSOJSLUqhZ/a8w+kiIRuQb4BXCpu83uXDLGnJJ20WGc36cD5/fpAEBxSSnr9x5yOvHd+1pmr90LOFVLv+S2ziWxNOeSWOc4q1oCVX0qkr7Ab4CFqvqqOwnjT6tbNjdQWUViTPPw3eHCir6W7XmszC3gWJFTtbSPCS+fmDIrLY5+yVa1+JpPpkgRkXigi6qu9Ca4pmaJxJjmqbiklG++PVRplckd3x8FIDRY6JccW2mEWHJcpJ8jblkas49kHvBDnMtgy3EWmpqvqrc3QpxNwhKJMS3H/kOFbl9LvlO17MrneJGzokTHthHlSSUrLZ5+yW0JD7GqpaEas48kVlUPuuuA/FtV7xGRZlWRGGNajqSYcC7o15EL+nUEoKiklHV7DpZXLEt35PHRqm8BCAsOon/ntuWJJSs1no6xEf4Mv0WqTyIJEZFOwFXAn3wcjzHGnJLQ4CAGpjj3p/xyuLNt38Hj5XfiL9mex4uLtvPc51sBSI6NYHBaxfLFfTu1JSwkyI9n0PzVJ5Hci3P/yBequlhEuuGx7K0xxgSa9m0jGN2/I6P7O1XLieJS1pZXLc6U+h+u3ANAeEgQAzrHlo8Oy0qNp31bq1pOhc21ZYxplfYePF5p+eJVuQWcKHH6WjrHRZYnliFp8fTp1JbQ4NZXtTRmZ3sK8C9gOKDA58AtqprbGIE2BUskxpi6FBaXsGa3U7WUDUHeU3AcgIjQIAZ2jmNwWsViYEkx4X6O2PcaM5HMAV4BXnI3XQv8TFVHeR1lE7FEYoxpiD0Fx1i6vWIOsTW7DpZXLV3aRZYnlSFp8fTuGENIC6taGnWuLVXNrGtbILNEYoxpDMeLKqqWsuSy92AhAJGhwQxMiS0fHZaVGkdCm+ZdtTTm8N/vRORa4FX3+TXAgXoGMRp4HAgGnqvubngRuQqYjHPZbIWqjnW3/w242G12n6q+7m7vCrwGtAOWAuNU9UR94jHGGG9EhAYzJM2pQABUld0FlftanluwhaIS5wt6WkJUeVIZnNoyqxaoX0WSCkwBhuH8sv8SuFlVd9TxumBgAzAKyAUWA9eo6lqPNhnAG8B5qponIu1VdZ+IXAzcCowBwoH5bpuDIvIG8I6qviYiT+Mkn6dqi8UqEmNMUzleVMLqXQXuCpNOctl/yKlaosKCGZQSV37T5ODUeNpFh/k54po1WkXiJowfVjn4rcBjdbz0dGCTqm5xX/MacBmw1qPNeOAJVc1z32ufu70vzt3zxUCxiKwARovIm8B5wFi33Qs41UyticQYY5pKRGgw2entyE5vBzhVS27esfJhx0t35PHM/C0Ulzpf4rsmRjM4taITv1fHGIKDmtfklA1dIfF26k4knYGdHs9zgaFV2vQEEJEvcC5/TVbVT4AVwD0i8ggQBYzESUAJQL6bYMqO2bm6NxeRCcAEgNTU1PqdlTHGNDIRoUu7KLq0i+KyTOfX1bETJazaVVA+h9hnG/bzztJdAESHBZPpkVgGp8YRFxW4VQs0PJHUJ11W16bqdbQQIAMYAaQAC0Skv6rOFpHTcC6j7QcWAsX1PKazUXUqMBWcS1v1iNcYY5pEZFgwp3dtx+ldK6qWnd8fK+/AX7ojjyfnbabErVq6JUWXJ5astDgy2gdW1dLQRFKfX8y5QBeP5ynA7mraLFLVImCriKzHSSyLVfV+4H6OQgVPAAAX4klEQVQAEXkF527674A4EQlxq5LqjmmMMc2KiJCaEEVqQhQ/GuxULUdPFLMyt6B8IbBPv9nHW0uc2/diwkPIdDvws1LjGNwlntgo/y0TVWMiEZFDVJ8wBKjPXM2LgQx3lNUu4Goq+jbKzMAZBTZdRBJxLnVtcTvq41T1gIgMxFknfraqqojMBa7EGbn1C+C9esRijDHNSlRYCGd0S+CMbgmAU7VsP3C0omrZns+UTzfiFi30aN+mfIqXrLR4eiS1IaiJqhafTpEiIhfh9KUEA9NU9X4RuRfIUdWZ4ix39jAwGigB7ndHY0XgDO0FOAj8RlWXu8fsRsXw32XAtapaWFscNmrLGNMSHSksZkVuvtOJ744SyztaBEBMRAiZXeK477L+pCdGN+j4PlnYqrmyRGKMaQ1Ula3fHSmfTn/p9jxeHX8G8Q0cYtyYNyQaY4xpBkSEbklt6JbUhiuHpDTZ+7a8WyyNMcY0KUskxhhjvFLnpa0aRm8VADnA78vuXDfGGNM61aeP5BGcezVewRn6ezXQEVgPTMO5mdAYY0wrVZ9LW6NV9RlVPaSqB907xi9yZ+ON93F8xhhjAlx9EkmpiFwlIkHu4yqPfS1/7LAxxpha1SeR/AwYB+xzH+OAa0UkEpjow9iMMcY0A/WZRn4LcGkNuz9v3HCMMcY0N3VWJCKSIiLvisg+EdkrIm+LSNPd6WKMMSag1efS1r+BmUAyztof77vbjDHGmHolkiRV/beqFruP6UCSj+MyxhjTTNQnkXwnIteKSLD7uBY44OvAjDHGNA/1SSTXA1cB3wJ7cNYCuc6XQRljjGk+6kwkqrpDVX+oqkmq2l5VfwT8uAliM8YY0ww0dNLG2xs1CmOMMc1WQxNJ4Kw6b4wxxq8amkhsahRjjDFALXe21zB9PDjVSKTPIjLGGNOs1JhIVDWmKQMxxhjTPPl0hUQRGS0i60Vkk4jcWUObq0RkrYisEZFXPLb/3d22TkT+KSLibr9GRFaJyEoR+UREEn15DsYYY2rns0QiIsHAE8AYoC9wjYj0rdImA7gLGK6q/YBb3e1nAsOBgUB/4DTgXBEJAR4HRqrqQGAlNgOxMcb4lS8rktOBTaq6RVVPAK8Bl1VpMx54QlXzAFR1n7tdgQggDAgHQoG9OP0zAkS7FUpbnNUbjTHG+IkvE0lnYKfH81x3m6eeQE8R+UJEFonIaABVXQjMxbmTfg8wS1XXqWoRcCOwCieB9AWer+7NRWSCiOSISM7+/fsb87yMMcZ48GUiqe5ek6qjwEKADJx1368BnhOROBHpAfQBUnCSz3kico6IhOIkksE4sxGvxLk0dvIbqU5V1WxVzU5KsjkmjTHGV+pc2MoLuUAXj+cpnHwZKhdY5FYaW0VkPRWJZZGqHgYQkY+BM4BjAKq62d3+BlBtJ74xxpim4cuKZDGQISJdRSQMuBpnXRNPM4CRAO7oq57AFmAHbue6W4WcC6wDdgF9RaSsxBjlbjfGGOMnPqtIVLVYRCYCs4BgYJqqrhGRe4EcVZ3p7rtARNYCJcAkVT0gIm8B5+H0hSjwiaq+DyAifwU+E5EiYDvwS1+dgzHGmLqJasuf7SQ7O1tzcnL8HYYxxjQrIrJEVbPraufTGxKNMca0fJZIjDHGeMUSiTHGGK9YIjHGGOMVSyTGGGO8YonEGGOMVyyRGGOM8YolEmOMMV6xRGKMMcYrlkiMMcZ4xRKJMcYYr1giMcYY4xVLJMYYY7xiicQYY4xXLJEYY4zxiiUSY4wxXrFEYowxxiuWSIwxxnjFEokxxhivhPg7gID2wW3w3UZIHgydhziP2BQQ8XdkxhgTMHyaSERkNPA4EAw8p6oPVtPmKmAyoMAKVR3rbv87cDFO1TQHuEVVVUTCgCnACKAU+JOqvu2TE2ibDLuXw1dPQ8kJZ1t0kpNQkrPc5JIFUe188vbGGNMc+CyRiEgw8AQwCsgFFovITFVd69EmA7gLGK6qeSLS3t1+JjAcGOg2/Rw4F5gH/AnYp6o9RSQI8N1v8XMmOY/iQti7GnYtdR67l8KGWTi5D4hPr5xYOg2CsGifhWWMMYHElxXJ6cAmVd0CICKvAZcBaz3ajAeeUNU8AFXd525XIAIIAwQIBfa6+64HervtS4HvfHgOjpDwiktbZY4fhD3L3eSyBHIXw5p3nH0SBO37elwSy3KeB4f6PFRjjGlqvkwknYGdHs9zgaFV2vQEEJEvcC5/TVbVT1R1oYjMBfbgJJIpqrpOROLc190nIiOAzcBEVd1b5biIyARgAkBqamrjnVWZiLbQ9RznUebwvorEsnspfPMBLHvJ2RcSAR0HViSWzkOgXTfrbzHGNHu+TCTV/YbUat4/A6e/IwVYICL9gUSgj7sNYI6InINTzaQAX6jq7SJyO/AQMO6kN1KdCkwFyM7Orvq+vtGmPfQa7TycICBvm5NYyi6JLZkOXz3l7I+IrXxJrPMQiOnYJKEaY0xj8WUiyQW6eDxPAXZX02aRqhYBW0VkPRWJZZGqHgYQkY+BM4AFwFHgXff1bwK/8tUJeE0E2nV1HgOudLaVFMP+byqqll1L4PNHQUuc/THJblJxE0vyYCfhGGNMgPJlIlkMZIhIV2AXcDUwtkqbGcA1wHQRScS51LUF6AaMF5EHcCqbc4HH3FFb7+Mkmk+B86nc5xL4gkOgY3/nMeQXzrYTR+HbVRWJZdcS57JYmYQMj8SSBR0HQGiEf+I3xpgqfJZIVLVYRCYCs3D6P6ap6hoRuRfIUdWZ7r4LRGQtUAJMUtUDIvIWcB6wCudy2Ceq+r576DuAl0TkMWA/cJ2vzqHJhEVB6lDnUebo97B7mZtclsKWebDydWdfUAh06F85uST1gqBgv4RvjGndRLVpug/8KTs7W3NycvwdhndU4eBuj6plqZNoCg86+0OjITmzcnKJS7XOfGNMg4nIElXNrqud3dneXIhAbGfn0edSZ1tpKXy/ueJy2K6l8NUzFTdPRiVWTiydsyA60X/nYIxpkSyRNGdBQZCY4TwGXe1sKz4B+9a4iWWZ8+fGOZQPmItLrXxnfqdBEN7Gb6dgjGn+LJG0NCFhzkiv5MFwmrut8BDsWeFx8+QSWOMOfJMgSOrtVCtlyaVDP7t50hhTb5ZIWoPwGEg/y3mUOby/oiN/1xJY/zEs+z9nX3A4dBpY+R6Xdt2dCsgYY6qwznbjUIX87R535i9zHkVHnf3hsdB5cOXk0jbZvzEbY3zKOtvNqRFxJp+MT4f+P3a2lZbA/vUVnfm7l8KX/4TSYmd/TKeKTvzOWc7ltMh4f52BMcZPLJGYmgUFQ4e+ziPLnYWm6Bh8u7rynfnrP6x4Tbvulad86TgAQiP9E78xpklYIjGnJjQSupzmPMocy6988+S2BbDqDWdfUIgz87FncknqbTdPGtOCWCIx3ouMg+4jnUeZg7srJqrctQRWvwNL/u3sC42CTpkel8SynEtqdvOkMc2SJRLjG22TnUefS5znpaXw/ZbKd+Z//SyUFDr7I9tVrlqSs6BNkv/iN8bUmyUS0zSCgiCxh/MYeJWzraQI9q7xSC7LYPN/QUud/bGpzkixssSSnOkMZTbGBBRLJMZ/gkOd5JCcCdnXO9sKD8O3Kyuqll1LYO177gvEmZwyebDTqR+f5typH5cGbTrYfS7G+IklEhNYwttA2pnOo8yRA5Vvntw8F1a8Wvl1weEQ18VJKnGpbpJxH/FpEJVgfTDG+IglEhP4ohMgY5TzKFN0DPJ3OjdR5m+HPPfP/B3OCLJj31c+Rmi0R4JJrUgwZT9HxmGMaRhLJKZ5Co2EpJ7OozqFh5ykkucml/JkswO2f1kx/X6ZiNiKpFKeZMoSTapNbGlMLSyRmJYpPMaZfLJDv5P3qcKxvJMTTP52OLAJNv0Xio9Vfk1UwslVTFmyie1iK1aaVs0SiWl9RCCqnfNIzjx5vyoc2e9WNNsqLpnlbYc9K+GbDyvWfCkT08mjokmtXNHEpthsyqZFs0RiTFUi0Ka980ipZr660lI4tKf6imbHIlj9VsUQZnCm6m+bUnMfTUwnu9PfNGuWSIw5VUFBFatVpg07eX9JERzc5dFH41HRbP7USUKVjhfqjjircsmsrKJp095GnJmAZonEmMYWHFoxk3LXavYXF1Yz4sytaL75EI5+V7l9SGRFp7/nJbOynyPjLdEYv/JpIhGR0cDjQDDwnKo+WE2bq4DJOGvBrlDVse72vwMXA0HAHOAW9Vg8RURmAt1Utb8vz8GYRhcSXnGXf3VOHDl5xFlZwsn9Go4XVG4fFlPzsOa4VIho6/tzMq2azxKJiAQDTwCjgFxgsYjMVNW1Hm0ygLuA4aqaJyLt3e1nAsOBgW7Tz4FzgXnu/h8Dh30VuzF+FRYN7fs4j+ocy/dIMB4JJ28bbJkPRUcqt4+Mr/lGzdguEBbl81MyLZsvK5LTgU2qugVARF4DLgPWerQZDzyhqnkAqrrP3a5ABBAGCBAK7HWP0wa4HZgAvOHD+I0JTJFxzqPTwJP3qcLR7yF/W5WKZgfsWwcbZlVMlFkmun3NFU1sFwgJa5LTMs2XLxNJZ2Cnx/NcYGiVNj0BROQLnMtfk1X1E1VdKCJzgT04iWSKqq5zX3Mf8DBwtLY3F5EJOMmG1NRUL0/FmGZCxJkJIDrBmeyyqtJSOLLPYxCAx6wAZfOala2A6RzQmcW5umHN8WkQkwzB1tXa2vnyf0B1vX9VF4gPATKAEUAKsEBE+gOJQB93G8AcETkHOAj0UNXbRCS9tjdX1anAVHDWbG/YKRjTwgQFQUxH55Fa9XsdUFLsDm3efnJFs+1zWPk6lT7GQSHQtrNHFZNuk2m2Qr5MJLlAF4/nKcDuatosUtUiYKuIrKcisSxS1cMAIvIxcAZwCBgiItvc2NuLyDxVHeHD8zCm9QgOcYcid4H0s07eX3wCDuaePKw5fztsnAOH91Y5nk2m2Rr4MpEsBjJEpCuwC7gaGFulzQzgGmC6iCTiXOraAnQDxovIAziVzbnAY6r6PvAUgFuRfGBJxJgmFBIG7bo5j+p4TqaZt63yTZs2mWaL5bNEoqrFIjIRmIXT/zFNVdeIyL1AjqrOdPddICJrgRJgkqoeEJG3gPOAVTh19CduEjHGBLK6JtM8fhAKdlZf0Wz7Ak4cqtzeczLN+PSTp6EJi/b5KZm6icetGS1Wdna25uTk+DsMY0xtyifT3H7yfTRlP580mWZizTdq2mSaXhORJapazTxBldlwC2NMYKg0mebgk/eXTabpOeKsfDLNFbDuAygtqvwaz8k0q96oaZNpNhpLJMaY5sFzMs0up528v3wyzWoqmnpNplmloonpaJNp1pMlEmNMy1BpMs0zT95fNplmdZfMTnUyzfg0iE6yEWcuSyTGmNbBczLN6hQdh4JcZ1aAqjM322SatbJEYowx4HTMn+pkmmVDnE91Ms34NGcVzxbCEokxxtRHgybT3N6wyTTjUp2h1M2EJRJjjGkMDZlMM297i5hM0xKJMcb4Wn0m0zy89+T1Z+ozmWbVYc3xac78Z0044swSiTHG+FtQELTt5DxqnExzd/U3am5d4IxGq2kyzUsfr3lKm0ZiicQYYwJdcEjFCLGaJtMs2Fl9H02o76eRsURijDHNXUgYJHR3Hn5gCwUYY4zxiiUSY4wxXrFEYowxxiuWSIwxxnjFEokxxhivWCIxxhjjFUskxhhjvGKJxBhjjFdaxZrtIrIf2O7vOFyJwHd1tvKvQI8x0OMDi7ExBHp8EPgxehtfmqom1dWoVSSSQCIiOaqa7e84ahPoMQZ6fGAxNoZAjw8CP8amis8ubRljjPGKJRJjjDFesUTS9Kb6O4B6CPQYAz0+sBgbQ6DHB4EfY5PEZ30kxhhjvGIViTHGGK9YIvERERktIutFZJOI3FlDm6tEZK2IrBGRVwItRhFJFZG5IrJMRFaKyEVNHN80EdknIqtr2C8i8k83/pUikhVg8f3MjWuliHwpIoOaMr76xOjR7jQRKRGRK5sqNvd964xPREaIyHL3czK/KeNz37+uf+dYEXlfRFa4MV7XxPF1cT+n69z3v6WaNr79rKiqPRr5AQQDm4FuQBiwAuhbpU0GsAyId5+3D8AYpwI3uj/3BbY1cYznAFnA6hr2XwR8DAhwBvBVgMV3pse/75imjq8+MXr8X/gU+Ai4MpDiA+KAtUCq+7xJPyf1jPF/gL+5PycB3wNhTRhfJyDL/TkG2FDNZ9mnnxWrSHzjdGCTqm5R1RPAa8BlVdqMB55Q1TwAVd0XgDEq0Nb9ORbY3YTxoaqf4Xwoa3IZ8KI6FgFxItKpaaKrOz5V/bLs3xdYBKQ0SWCVY6jr7xDgJuBtoKn/D9YnvrHAO6q6w20fiDEqECMiArRx2xY3RWwAqrpHVZe6Px8C1gGdqzTz6WfFEolvdAZ2ejzP5eR/2J5ATxH5QkQWicjoJovOUZ8YJwPXikguzrfVm5omtHqrzzkEil/hfCMMKCLSGbgceNrfsdSgJxAvIvNEZImI/NzfAVVjCtAH54vWKuAWVS31RyAikg4MBr6qssunnxVbs903pJptVYfHheBc3hqB8011gYj0V9V8H8dWpj4xXgNMV9WHRWQY8JIbo18+JNWozzn4nYiMxEkkZ/k7lmo8BtyhqiXOF+qAEwIMAc4HIoGFIrJIVTf4N6xKLgSWA+cB3YE5IrJAVQ82ZRAi0gansry1mvf26WfFEolv5AJdPJ6ncPJloVxgkaoWAVtFZD1OYlncNCHWK8ZfAaMBVHWhiETgzN3T5JcXalCfc/ArERkIPAeMUdUD/o6nGtnAa24SSQQuEpFiVZ3h37DK5QLfqeoR4IiIfAYMwukHCBTXAQ+q0xmxSUS2Ar2Br5sqABEJxUkiL6vqO9U08elnxS5t+cZiIENEuopIGHA1MLNKmxnASAARScQp4bcEWIw7cL4JIiJ9gAhgfxPGWJeZwM/dESlnAAWqusffQZURkVTgHWBcgH2DLqeqXVU1XVXTgbeA3wZQEgF4DzhbREJEJAoYitMHEEg8PycdgF404WfZ7Zt5Hlinqo/U0MynnxWrSHxAVYtFZCIwC2dEzDRVXSMi9wI5qjrT3XeBiKwFSoBJTfmNtZ4x/h54VkRuwymDf+l+62oSIvIqzqW/RLef5h4g1I3/aZx+m4uATcBRnG+GTaYe8f0FSACedL/xF2sTT/BXjxj9qq74VHWdiHwCrARKgedUtdahzE0dI3AfMF1EVuFcQrpDVZtyRuDhwDhglYgsd7f9D5DqEaNPPyt2Z7sxxhiv2KUtY4wxXrFEYowxxiuWSIwxxnjFEokxxhivWCIxxhjjFUskxhhjvGKJxJgA4U6X/oG3bYxpapZIjKkn965g+8wYU4V9KIyphYikuwsGPQksBcaJyEIRWSoib7oT5SEiF4nINyLyubuAUI1Vg4icLs5CV8vcP3tV02ayiLwkIp+KyEYRGe+xu42IvOW+38vuFBmIyF9EZLGIrBaRqWXbjfE1SyTG1K0X8CIwCmciyx+oahaQA9zuTmb5DM7EjGfhLG5Um2+Ac1R1MM40Kv+vhnYDgYuBYcBfRCTZ3T4YuBVnsbFuOFNkAExR1dNUtT/OTLmXnPKZGtMANteWMXXbrqqLROQSnF/eX7hf9sOAhTgzvW5R1a1u+1eBCbUcLxZ4QUQycOYwC62h3Xuqegw4JiJzcRYjywe+VtVcAHdupXTgc2CkiPwRiALaAWuA9xt2ysbUnyUSY+p2xP1TgDmqeo3nThEZfIrHuw+Yq6qXuwsRzauhXdWJ8MqeF3psKwFC3KroSSBbVXeKyGSc2ZqN8Tm7tGVM/S0ChotIDwARiRKRnjiXqrq5SQHgp3UcJxbY5f78y1raXSYiESKSgDP7bG1r1ZQlje/cfpsr64jBmEZjicSYelLV/Ti/+F8VkZU4iaW3e/npt8AnIvI5sBcoqOVQfwceEJEvcKbwr8nXwIfu+9ynqjUuROSurPkszlKvM2i6BdKMsWnkjWkMItJGVQ+7I6WeADaq6qNeHG8ycFhVH2qsGI3xFatIjGkc492O7zU4l66e8XM8xjQZq0iM8RERuQ64pcrmL1T1d/6IxxhfsURijDHGK3ZpyxhjjFcskRhjjPGKJRJjjDFesURijDHGK5ZIjDHGeOX/A8gRUJWpZiEYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (-gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'test_reg_alpha:'   + str(value))\n",
    "# for i, value in enumerate(reg_alpha):\n",
    "#     pyplot.plot(reg_lambda, -train_scores[i], label= 'train_reg_alpha:'   + str(value))\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当前调参结果：  \n",
    "最优reg_alpha为2；   \n",
    "最优reg_lambda为2。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
